<?php
if (!defined('IN_SYS')) {
    exit('Access Denied');
}

class catch_controller extends controller
{
    public function __construct()
    {
        parent::__construct();
    }
    
    public function caijinews()
    {
        
        $lockfile = DATA.'/cache/lock_auto.time';
        $mintime = is_file($lockfile)?filemtime($lockfile):0;
//        if(!IS_AJAX && time()-$mintime<(100-$GLOBALS['G']['speed'])*9){
//            exit;
//        }
        touch($lockfile, time());

        $data = array();
        $domain = array();
        $toUrl = REMOTEAPI."/dataapi/allinone";
        $yuming = db::select("select id,name from yuming where autoCatch = 1 order by id asc");
        if (empty($yuming)) {
            exit;
        }
        foreach ($yuming as $value) {
            $domain[$value['id']] = $value['name'];
        }
        $site_ym = join(',', $domain);
        $data['http_host'] = $_SERVER["HTTP_HOST"];
        $data['appid'] = $GLOBALS['G']['appid'];
        $data['domain']=$site_ym;
        $respond = yumi::curlpost($toUrl, $data, $GLOBALS['G']['appkey']);
        if (empty($respond)) {
            exit("通信失败");
        }
        $respond = json_decode($respond['html'], true);
        if (empty($respond)) {
            exit("通信失败");
        } elseif ($respond['status']!=1) {
            echo $respond['errMsg'];
            exit();
        }

        $ymids = array_flip($domain);
        //打乱域名id顺序
        $num = 0;
        $ids = array();
        $arcdata = array();
        foreach ($respond['data'] as $key=>$value) {
            $yumingname = addslashes($value['yuming']);
            $yuming_id = $value['yuming_id'] = $ymids[$yumingname];
            
            $title = addslashes($value['title']);
            $res = db::find("select id from `content_{$yuming_id}` where title='$title'");
            
            if (empty($res)) {
                $arcdata[] = $value;
                $ids[] = $value['id'];
                $num++;
            }
        }
        //dump($respond['data']);
        if ($num>0) {
            $this->saveNews($arcdata, $ids);
        }
    }

    private function saveNews($datas, $ids = array())
    {
        if (!$ids) {
            echo "成功采集到0篇新闻";
            exit();
        }
        $toUrl = REMOTEAPI."/dataapi/content";
        $data['http_host'] = $_SERVER["HTTP_HOST"];
        $data['appid'] = $GLOBALS['G']['appid'];
        $data['aidstr'] = implode(',', $ids);
        $respond = yumi::curlpost($toUrl, $data, $GLOBALS['G']['appkey']);
        if (empty($respond)) {
            exit("通信失败");
        }
        $respond = json_decode($respond['html'], true);
        //dump($respond);
        //exit;
        if (empty($respond)) {
            exit("通信失败");
        } elseif ($respond['status']!=1) {
            echo $respond['errMsg'];
            exit();
        }

        $filecontent_list = $respond['data'];
        $num = 0;
        foreach ($datas as $key=>$value) {
            $title = addslashes($value['title']);
            $source = addslashes($value['source']);
            $yumingname = addslashes($value['yuming']);
            $yuming_id = $value['yuming_id'];
                
            //过滤文章内容
            $aid = $value['id'];

            $filecontent = str_replace(array("<p>(点击图片看大图)</p>","显示全文"), "", $filecontent_list[$aid]);

            //过滤原标题
            $filecontent = str_replace("原标题：", "", $filecontent);
            $filecontent = preg_replace("/20\d{2}-\d{2}-\d{2} \d{2}:\d{2}/i", "", $filecontent);

            $filecontent = str_replace("</p></p>", "</p>", $filecontent);
            $filecontent = str_replace("<p><p align=\"center\">", "<p align=\"center\">", $filecontent);
            $filecontent = preg_replace("/上一页.*?下一页/", "", $filecontent);

            //获取描述内容
            $description=strip_tags($filecontent);
            $description=trim($description);
            $description=str_replace(array(" "," "), "", $description);
            $description=mb_substr($description, 0, 150, 'utf-8');
            $description=addslashes($description);

            //图片处理
            $filecontent=yumi::convert_images_patch($filecontent,'',$toUrl, $yuming_id);
            // 标题自动优化
            $seo_keyword=yumi::autoseo($title, $yuming_id);

            //同义词替换
            $filecontent=yumi::replace_word($filecontent, $yuming_id);
            //关键词优化
            $seod['title']=$title;
            $seod['body']=$filecontent;
            $seod=yumi::seoword($seod, $yuming_id);
            $title=$seod['title'];
            $filecontent=$seod['body'];
            //取出图片存入数据库
            if (preg_match('#src="([^"]*)"#', $filecontent, $nt)) {
                $thumb = $nt[1];
            } else {
                $thumb="";
            }

            $addtime=time();

            //文章标记
            $flag = '';
            if ($addtime%7==0) {
                $flag = "h";   //头条
            } elseif ($addtime%11==0) {
                $flag = "c";   //推荐
            } elseif ($addtime%17==0) {
                $flag = "a";   //推荐
            } elseif ($thumb && $addtime%3==0) {
                $flag = "f";   //推荐
            }

            db::query("INSERT into `content_{$yuming_id}`(title,flag,source,thumb,class,status,description,addtime,seo_keyword)  values ('$title','$flag','$source','$thumb','$value[class]','$value[status]','$description','$addtime','$seo_keyword')");
            $id=db::insert_id();
            //将文章内容存入文件
            ymcms::article_write($yuming_id, $id, $filecontent);
            $num++;
        }
        echo "成功采集到".$num."篇新闻";
    }

    //规则采集：自动采集文章
    public function caijiArticle()
    {
        $yumingid = gp("yumingid");
        $this->yumingid = $yumingid?intval($yumingid):$this->yuming_website['id'];

        $lockfile = DATA.'/cache/lock_'.$this->yumingid.'.time';
        $mintime = is_file($lockfile)?filemtime($lockfile):0;
        if (!IS_AJAX && time()-$mintime<(100-$GLOBALS['G']['speed'])*3) {
            exit;
        }

        touch($lockfile, time());

        if (!IS_AJAX) {
            if (!db::find("select * from yuming where autoRule=1 AND id=".$this->yumingid)) {
                exit;
            }
        }

        //取出文章链接
        $url = db::find("select * from catchlinks where status=1 AND (yuming_id=".$this->yumingid." OR yuming_id=0) order by id asc");
        if (empty($url)) {
            $this->CatchLinks();
            exit;
        }

        //取出规则
        $res=db::find("select * from catch where id='$url[ruleid]'");
        $result=httpUnit::get($url['link']);
        $data="";
        if ($result['httpcode']==200) {
            $data=$result['html'];
            //更改文章状态
            db::query("update catchlinks set status=0 where id='$url[id]'");
        } else {
            db::query("update catchlinks set status=2 where id='$url[id]'");
            echo $url['link']."无法采集此文章";
            exit;
        }

        //判断编码
        $unicode=htmlBase::get_charset($data);
        //转换编码
        if (!empty($unicode) && $unicode!=="utf-8") {
            $data=mb_convert_encoding($data, "utf-8", $unicode);
        } else if (strrpos($url['link'], 'yuanlin.com')) {
            $data = mb_convert_encoding($data,"utf-8",'gbk');
        }

        //匹配关键词
        $keywords=htmlBase::get_keywords($data);

        //匹配描述
        $description=htmlBase::get_description($data);

        //匹配文章标题
        $title = htmlHelper::matchcontent($data, $res['titlerule']);
        $title= str_replace(array("\r\n", "\r", "\n"), "", $title);
        $title = strip_tags($title);

        //匹配文章内容
        $body = htmlHelper::matchcontent($data, $res['contentrule']);
        $this->SaveArticles($title, $body, $keywords, $description, $url['yuming_id'], $url['id'], $res['class'], $url['link']);
    }

    //存入文章
    private function SaveArticles($title, $body, $keywords, $description, $yi, $ui, $class, $link='')
    {
        if (empty($title)||empty($body)) {
            db::query("update catchlinks set status=2 where id='$ui'");
            echo "空标题或者空内容";
            exit;
        }

        //获取域名
        if ($yi==0) {
            //随机获取域名
            $yuming=db::select("select id from yuming order by id asc");
            //打乱域名顺序
            shuffle($yuming);
            $key= array_rand($yuming, 1);
            $yuming_id=  $yuming[$key]['id'];
        } else {
            $yuming_id=$yi;
        }

        //文章查重
        $title = trim($title);
        $article=db::find("select * from `content_{$yuming_id}` where title='$title'");

        if ($article) {
            db::query("update catchlinks set status=3 where id='$ui'");
            echo "采集到重复文章跳过：".$title;
            exit;
        } else {
            //处理a标签
            $filecontent = preg_replace('#<[/]?a[^>]*?>#', "", $body);
            //过滤div标签
            $filecontent = preg_replace('#<[/]?div[^>]*?>#', "", $filecontent);
            //过滤空
            $filecontent = preg_replace('#<([a-z]*?)\s[^>]*></\\1>#', "", $filecontent);
            
            //图片处理，传入采集URL，构建图片完整路径
            $filecontent = yumi::convert_images_patch($filecontent, '', $link); 

            //过滤文章内容
            $filecontent=htmlBase::tag_filter($filecontent);
            //过滤data-original
            // $filecontent = preg_replace('/data-original=\"(.*?)\"/is', '',$filecontent);

            //获取栏目
            if ($class=='') {
                $classids=db::select("select ename from classlist where yuming_id='$yuming_id'");
                shuffle($classids);
                $key= array_rand($classids, 1);
                $class = $classids[$key]['ename'];
            }

            // 标题自动优化
            $seo_keyword = yumi::autoseo($title, $yuming_id);

            //同义词替换
            $filecontent = yumi::replace_word($filecontent, $yuming_id);

            //关键词优化
            $seod['title']=$title;
            $seod['body']=$filecontent;
            $seod=yumi::seoword($seod, $yuming_id);
            $title=$seod['title'];
            $filecontent=$seod['body'];

            //取出图片存入数据库
            if (preg_match('#src="([^"]*)"#', $filecontent, $nt)) {
                $thumb = $nt[1];
            } else {
                $thumb = '';
            }
            $addtime = time();

            //文章标记
            $flag = '';
            if ($addtime%7==0) {
                $flag = "h";   //头条
            } elseif ($addtime%11==0) {
                $flag = "c";   //推荐
            } elseif ($addtime%17==0) {
                $flag = "a";   //推荐
            } elseif ($thumb && $addtime%3==0) {
                $flag = "f";   //推荐
            }

            $title = strip_tags($title);
            $title = addslashes($title);
            $description = addslashes($description);
            $keywords = addslashes($keywords);
            $seo_keyword = addslashes($seo_keyword);
            db::query("insert into `content_{$yuming_id}`(title,flag,source,thumb,status,description,addtime,class,keyword,seo_keyword)  values ('$title','$flag','','$thumb','1','$description','$addtime','$class','$keywords','$seo_keyword')");
            $id=db::insert_id();

            //将文章内容存入文件
            ymcms::article_write($yuming_id, $id, $filecontent);
            echo $title."文章采集成功";
        }
    }

    //规则采集
    private function CatchLinks()
    {
        $condition = "";
        if (!IS_AJAX && $this->yumingid>0) {
            $condition = " AND yuming_id=0 OR yuming_id=".$this->yumingid;
        }
        //采集文章网址
        $res=db::find("select * from catch where 1=1 $condition order by updatetime asc ");
        if (!$res) {
            exit();
        }
        
        $url=$res['listurl'];
        $page=$res['page'];
        $updatetime=time();
        db::query("update catch set updatetime='$updatetime' where id='".$res['id']."'");
        if (preg_match("#\[([1-9]\d*)\-([1-9]\d*)\]#", $url, $mt)) {
            if ($page<$mt[1]) {
                $page++;
            } elseif ($page<$mt[2]&&$page>=$mt[1]) {
                $page++;
            } elseif ($page = $mt[2]) {
                $page = $mt[1];
            }
            $url = preg_replace("#\[([1-9]\d*)\-([1-9]\d*)\]#", $page, $url);
            db::query("update catch set page='$page' where id='".$res['id']."'");
        }

        $result = httpUnit::get($url);
        $data = "";
        if ($result['httpcode']==200) {
            $data = $result['html'];
        } else {
            echo "无法采集此网站";
            echo $url;
            exit;
        }
        //判断编码
        $unicode = htmlBase::get_charset($data);
        //转换编码
        if (!empty($unicode) && $unicode!=="utf-8") {
            $data = mb_convert_encoding($data, "utf-8", $unicode);
        }

        //获取文章链接
        $html2 = htmlHelper::matchcontent($data, $res['articlerule']);
        $arr_link = array();
        if (preg_match_all("#href=[\"'\s]*([^\s\"'\\\\]*)[\"'\s]#", $html2, $mt3)) {
            $arr_link = $mt3[1];
        }
        if (!empty($res['norule'])) {
            $norule=explode("\n", $res['norule']);
            foreach ($norule as  $k => $value) {
                $norule[$k] = trim($value);
            }
            $norule=array_filter($norule);
            foreach ($arr_link as $k => $value) {
                foreach ($norule as $x => $y) {
                    if (stripos($value, $y)!==false) {
                        unset($arr_link[$k]);
                    }
                }
            }
        }
        $n=0;
        foreach ($arr_link as $key => $value) {
            if (strpos($value, 'javascript')!==0 && strpos($value, '#')!==0) {
                $link = htmlHelper::fix_path($res['listurl'], $value);
                $old=db::find("select * from catchlinks where  link='$link'");
                if (!$old) {
                    $n++;
                    db::query("insert into catchlinks(yuming_id,link,ruleid,status) values('$res[yuming_id]','$link','$res[id]','1')");
                }
            }
        }
        exit("采集到网址".$n."个");
    }

    // 采集测试，请勿删除
    public function test()
    {
        echo "server is ok";
    }
}
